package com.chance.cc.crawler.development.scripts.pcauto.forum;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.Downloader;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Html;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.CategoryTag;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.text.ParseException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.*;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.*;
import static com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField.*;

/**
 * @author lt
 * @version 1.0
 * @date 2021-01-14 11:36:19
 * @email okprog@sina.com
 */
public class PcAutoForumReplyCrawlerScript extends CrawlerCommonScript {

    private Logger logger = LoggerFactory.getLogger(PcAutoForumReplyCrawlerScript.class);

    private static final String indexRegex = "https?://www\\.pcauto\\.com\\.cn/";
    private static final String keysRegex = "https?://\\S*v1/meta/pcauto/keys\\S*";
    private static final String homeUrlRegex = "https?://price\\.pcauto\\.com\\.cn/\\S*/";
    private static final String listUrlRegex = "https:?//bbs\\.pcauto\\.com\\.cn/forum-\\d*-\\d*\\.html";
    private static final String articleUrlRegex = "https?://bbs\\.pcauto\\.com\\.cn/topic-\\d*-\\d*.html";
    private static final String viewsRegex = "https://bbs\\.pcauto\\.com\\.cn/intf/topic/counter\\.ajax\\S*";
    private static final String followsRegex = "https://my\\.pcauto\\.com\\.cn/bip/intf/focus\\.jsp\\S*";
    private static final String authorInfoRegex = "https://my\\.pcauto\\.com\\.cn/\\d*/forum/";
    private static final String authorCarInfoRegex = "https://bbs\\.pcauto\\.com\\.cn/topic/loadStaticInfos\\.ajax\\?vids=\\d*";
    private static final String commentRegex = "https://www\\.pcauto\\.com\\.cn/#\\d*";

    private static final String homeUlrFormat = "https://price.pcauto.com.cn/%s/";
    private static final String listUrlFormat = "https://bbs.pcauto.com.cn/forum-%s-%s.html";
    private static final String articleUrlFormat = "https://bbs.pcauto.com.cn/topic-%s-%s.html";
    private static final String viewsUrlFormat = "https://bbs.pcauto.com.cn/intf/topic/counter.ajax?tid=%s&fid=%s";
    private static final String followsUrlFormat = "https://my.pcauto.com.cn/bip/intf/focus.jsp?act=getFocusByNum&accountId=%s";
    private static final String authorInfoUrlFormat = "https://my.pcauto.com.cn/%s/forum/";
    private static final String authorCarInfoUrlFormat = "https://bbs.pcauto.com.cn/topic/loadStaticInfos.ajax?vids=%s";

    private static final String scriptSite = "forum_reply";

    private static final long ONE_SECOND = 1000L;
    private static final long ONE_MINUTE = 60000L;
    private static final long ONE_HOUR = 3600000L;
    private static final long ONE_DAY = 86400000L;

    @Override
    public String domain() {
        return "pcauto";
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(indexRegex);
        addUrlRegular(keysRegex);
        addUrlRegular(homeUrlRegex);
        addUrlRegular(listUrlRegex);
        addUrlRegular(articleUrlRegex);
        addUrlRegular(commentRegex);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        CategoryTag categoryTag = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag();
        String crawlerSite = categoryTag.getKVTagStrVal("site");
        return crawlerSite.equalsIgnoreCase(scriptSite);
    }

    @Override
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord requestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> allItemRecords = new ArrayList<>();
        if (supportSourceRecords == null || supportSourceRecords.size() < 1) {
            return super.prepareRequest(requestRecord, supportSourceRecords);
        }
        CrawlerRequestRecord keywordRecord = supportSourceRecords.get(0);
        String keywordUrl = keywordRecord.getHttpRequest().getUrl();
        if (keywordUrl.matches(keysRegex)) {
            try {
                JSONObject jsonObject = JSONObject.parseObject(keywordRecord.getInternalDownloadPage().getRawText());
                if (jsonObject.getIntValue("status") == 0) {
                    JSONArray objects = jsonObject.getJSONArray("content");
                    for (Object object : objects) {
                        String keyword = ((JSONObject) object).getString("keyword");
//                        keyword = "sg9306";
                        String homeUlr = String.format(homeUlrFormat, keyword);
                        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                                .turnPageRequest(requestRecord)
                                .recordKey(homeUlr)
                                .httpUrl(homeUlr)
                                .releaseTime(System.currentTimeMillis())
                                .copyBizTags()
                                .copyScheduleTags()
                                .notFilterRecord()
                                .build();
                        HttpRequest httpRequest = crawlerRequestRecord.getHttpRequest();
                        httpRequest.addHeader("Host", "price.pcauto.com.cn");
                        httpRequest.addHeader("Referer", "https://www.pcauto.com.cn/");
                        httpRequest.addHeader("User-Agent", getRandomUA());
                        httpRequest.addHeader("Cookie", "xdj_half_ab=a; u=50x2fzf; pcsuv=1610334270311.a.653977251; c=50x2lkg; locationWap=%7B%22isUser%22%3A0%2C%22pro%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%E7%9C%81%22%2C%22city%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22cityCode%22%3A%22310000%22%2C%22proCode%22%3A%22310000%22%2C%22cityUser%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22cityCodeUser%22%3A%22310000%22%2C%22proCodeUser%22%3A%22310000%22%2C%22regionPro%22%3A7%2C%22regionCity%22%3A3%2C%22url%22%3A%22%2F%2Fm.pcauto.com.cn%2Fx%2Fqcbj%2Fsh%2F%22%2C%22expires%22%3A1611639787528%7D; m_CMT4_IP_AREA=%E4%B8%8A%E6%B5%B7%E5%B8%82; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22176ef7676ff590-0ccbad136e030c-31346d-2073600-176ef7677008c5%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%22176ef7676ff590-0ccbad136e030c-31346d-2073600-176ef7677008c5%22%7D; UM_distinctid=1771f5052622e9-0033ca8f8137a9-31346d-1fa400-1771f5052638bc; PClocation=3; pcautoLocate=%7B%22proId%22%3A7%2C%22cityId%22%3A3%2C%22url%22%3A%22%2F%2Fwww.pcauto.com.cn%2Fqcbj%2Fsh%2F%22%2C%22dataTypeAuto%22%3A%22user%22%7D; pcLocate=%7B%22proCode%22%3A%22310000%22%2C%22pro%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%E7%9C%81%22%2C%22cityCode%22%3A%22310000%22%2C%22city%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22dataType%22%3A%22user%22%2C%22expires%22%3A1615260185126%7D; JSESSIONID=abcU-0uaxCiK36Bg9LWFx; iwt_uuid=45038466-41da-4ea3-bade-4d3baf4f0b34; __PCautoPrice4s_area_id_=3-%u4E0A%u6D77-3; 101362=%7B%22id%22%3A3225%2C%20%22name%22%3A%22%E6%9C%97%E9%80%B8%22%7D; captcha=102515f6402dde214758fd1b177f5a971c4-218e8108332185644046; favCar=%E4%B8%8A%E6%B1%BD%E5%A4%A7%E9%80%9AMAXUS%20V80%E6%96%B0%E8%83%BD%E6%BA%90_22696%7C%E9%A3%8E%E9%AA%8F%E6%88%BF%E8%BD%A6_12874%7C%E6%8E%A2%E5%B2%B3_22813%7C%E6%9C%97%E9%80%B8_3225%7C%E5%93%AA%E5%90%92U_24455; pc_browsing_history=%7B%22id%22%3A24455%2C%20%22name%22%3A%22%E5%93%AA%E5%90%92U%22%7D; ztLocate=%7B%22proId%22%3A7%2C%22cityId%22%3A3%2C%22proCode%22%3A%22310000%22%2C%22cityCode%22%3A%22310000%22%2C%22url%22%3A%22%2F%2Fwww.pcauto.com.cn%2Fqcbj%2Fsh%2F%22%2C%22proLetter%22%3A%22S%22%2C%22area%22%3A%22%E5%8D%8E%E4%B8%9C%22%2C%22near%22%3A%22%E5%8D%8E%E4%B8%9C%22%2C%22cityLetter%22%3A%22S%22%2C%22name%22%3A%22%E4%B8%8A%E6%B5%B7%22%2C%22pro%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22core%22%3Atrue%2C%22proPinyin%22%3A%22shanghaishi%22%2C%22pinyin%22%3A%22shanghai%22%2C%22city%22%3A%22%E4%B8%8A%E6%B5%B7%22%7D; pcuvdata=lastAccessTime=1614753324404|visits=151; channel=6609; visitedfid=31736D30795D18555D23855D16040D27445D21855D25455D15820D32236");
                        allItemRecords.add(crawlerRequestRecord);
//                        break;
                    }
                }

            } catch (Exception e) {
                logger.error(e.getMessage(), e);
            }
        }
        if (allItemRecords.isEmpty()) {
            return super.prepareRequest(requestRecord, supportSourceRecords);
        }
        return allItemRecords;
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList<CrawlerRequestRecord>();
        HttpRequest lastRequest = crawlerRequestRecord.getHttpRequest();
        int statusCode = httpPage.getStatusCode();
        String lastRequestUrl = lastRequest.getUrl();
        if (StringUtils.isBlank(httpPage.getRawText()) || !httpPage.isDownloadSuccess() || statusCode != 200) {
            Map<String, Object> extras = lastRequest.getExtras();
            int downloadTimes = 1;
            if (null == extras) {
                extras = new HashMap<>();
                extras.put("downloadTimes", downloadTimes);
                crawlerRequestRecord.getHttpRequest().setExtras(extras);
            } else {
                try {
                    downloadTimes = Integer.parseInt((String) extras.get("downloadTimes"));
                    extras.put("downloadTimes", downloadTimes + 1);
                } catch (Exception e) {
                    extras.put("downloadTimes", downloadTimes);
                }
            }
            if (statusCode == 404 || downloadTimes > 10 || (httpPage.getRawText() != null && httpPage.getRawText().contains("抱歉，"))) {
                logger.error("页面不存在：{},本页面下载次数：{}", statusCode, downloadTimes);
                return parsedLinks;
            }

            parsedLinks.add(crawlerRequestRecord);
            crawlerRequestRecord.setNeedWashPage(false);
            crawlerRequestRecord.tagsCreator().requestTags().removeRequestType(CrawlerEnum.CrawlerRequestType.filter);
            logger.error("页面下载状态：{}，状态码：{}，内容为空：{}，实行回推", httpPage.isDownloadSuccess(), statusCode, StringUtils.isBlank(httpPage.getRawText()));
            return parsedLinks;
        }
        if (lastRequestUrl.matches(homeUrlRegex)) {
            return parseHomeLinks(crawlerRequestRecord, httpPage, parsedLinks, lastRequestUrl);
        }
        if (lastRequestUrl.matches(listUrlRegex)) {
            return parseListLinks(crawlerRequestRecord, httpPage, parsedLinks);
        }
        if (lastRequestUrl.matches(articleUrlRegex)) {
            return parseArticleLinks(crawlerRequestRecord, httpPage, parsedLinks, lastRequest, lastRequestUrl);
        }
        if (lastRequestUrl.matches(commentRegex)) {
            HttpRequest httpRequest = crawlerRequestRecord.getHttpRequest();
            Map<String, Object> extras = httpRequest.getExtras();
            String authorId = (String) extras.get("authorId");
            //作者信息
            String authorUrl = String.format(authorInfoUrlFormat, authorId);
            String authorCarUrl = String.format(authorCarInfoUrlFormat, authorId);
            CrawlerRequestRecord authorInfoRecord = getAuthorInfoRecord(crawlerRequestRecord, lastRequestUrl, authorUrl);
            CrawlerRequestRecord authorCarInfoRecord = getAuthorCarInfoRecord(crawlerRequestRecord, lastRequestUrl, authorCarUrl);
            parsedLinks.add(authorInfoRecord);
            parsedLinks.add(authorCarInfoRecord);

            //作者粉丝数
            String followsUrl = String.format(followsUrlFormat, authorId);
            CrawlerRequestRecord followsRecord = getFollowsRecord(crawlerRequestRecord, authorUrl, followsUrl);
            parsedLinks.add(followsRecord);
            return parsedLinks;
        }
        return null;
    }

    private List<CrawlerRequestRecord> parseArticleLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks, HttpRequest lastRequest, String lastRequestUrl) {
        Map<String, Object> extras = lastRequest.getExtras();
        String forumKey = (String) extras.get("forumKey");
        String articleKey = (String) extras.get("articleKey");
        int articlePage = (int) extras.get("articlePage");
        if (1 == articlePage) {
            //判断帖子分页
            List<String> pagingList = httpPage.getHtml().xpath("//div[@id=\"pager\"]/a/@href | //div[@class=\"pcauto_page\"]/a/@href").all();
            if (null != pagingList && pagingList.size() > 0) {
                String maxPageUrl = pagingList.get(pagingList.size() - 2);
                int page = Integer.parseInt(maxPageUrl.substring(maxPageUrl.lastIndexOf("-") + 1).split("\\.")[0]);
                maxPageUrl = String.format(articleUrlFormat, articleKey, page);
                CrawlerRequestRecord maxPageRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .httpUrl(maxPageUrl)
                        .recordKey(maxPageUrl)
                        .releaseTime(crawlerRequestRecord.getReleaseTime())
                        .copyBizTags()
                        .notFilterRecord()
                        .build();
                Map<String, Object> maxPageExtras = copyExtras(extras);
                maxPageExtras.put("articlePage", page);
                maxPageExtras.put("downloadTimes", 1);
                HttpRequest maxPageRequest = maxPageRecord.getHttpRequest();
                maxPageRequest.setExtras(maxPageExtras);
                maxPageRequest.addHeader("Host", "bbs.pcauto.com.cn");
                maxPageRequest.addHeader("User-Agent", getRandomUA());
                parsedLinks.add(maxPageRecord);
            }

            //帖子views
            String viewsUrl = String.format(viewsUrlFormat, articleKey, forumKey);
            CrawlerRequestRecord viewsRecord = getViewsRecord(crawlerRequestRecord, lastRequestUrl, viewsUrl);
            parsedLinks.add(viewsRecord);

            String authorId = "";
            String authorUrl = null;
            try {
                authorUrl = httpPage.getHtml().xpath("//div[@id=\"post_list\"]/div[@id][1]/table/tbody/tr[1]/td[1]/div/p/a/@href").get();
                authorId = authorUrl.split("cn/")[1].split("/f")[0];
            } catch (Exception e) {
                try {
                    authorUrl = httpPage.getHtml().xpath("//div[@class=\"user-name\"]/a/@href").get();
                    authorId = authorUrl.split("cn/")[1].split("/t")[0];
                } catch (Exception ex) {
                    authorId = "";
                }
            }
            //作者信息
            authorUrl = String.format(authorInfoUrlFormat, authorId);
            CrawlerRequestRecord authorInfoRecord = getAuthorInfoRecord(crawlerRequestRecord, lastRequestUrl, authorUrl);
            String authorCarUrl = String.format(authorCarInfoUrlFormat, authorId);
            parsedLinks.add(authorInfoRecord);
            CrawlerRequestRecord authorCarInfoRecord = getAuthorInfoRecord(crawlerRequestRecord, lastRequestUrl, authorCarUrl);
            parsedLinks.add(authorCarInfoRecord);


            //作者粉丝数
            String followsUrl = String.format(followsUrlFormat, authorId);
            CrawlerRequestRecord followsRecord = getFollowsRecord(crawlerRequestRecord, authorUrl, followsUrl);
            parsedLinks.add(followsRecord);
        }
        //递减翻页
        if (articlePage > 2) {
            String nextPageUrl = String.format(articleUrlFormat, articleKey, (articlePage - 1));
            CrawlerRequestRecord nextPagReocrd = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .httpUrl(nextPageUrl)
                    .recordKey(nextPageUrl)
                    .releaseTime(crawlerRequestRecord.getReleaseTime())
                    .copyBizTags()
                    .build();
            Map<String, Object> nextPageExtras = copyExtras(extras);
            nextPageExtras.put("articlePage", (articlePage - 1));
            nextPageExtras.put("downloadTimes",1);
            HttpRequest maxPageRequest = nextPagReocrd.getHttpRequest();
            maxPageRequest.setExtras(nextPageExtras);
            maxPageRequest.addHeader("Host", "bbs.pcauto.com.cn");
            maxPageRequest.addHeader("User-Agent", getRandomUA());
            parsedLinks.add(nextPagReocrd);
        }

        //解析评论
        String commentNodes = httpPage.getHtml().xpath("//div[@id=\"J_mComment\"]/div//span/text()").get();
        if (null == commentNodes) {
            //正常帖子评论
            parseNormalComments(crawlerRequestRecord, httpPage, parsedLinks, articleKey);
        } else {
            //游记帖子评论
            parseTravelComments(crawlerRequestRecord, httpPage, parsedLinks, articleKey);
        }
        return parsedLinks;
    }

    private void parseTravelComments(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks, String articleKey) {
        List<Selectable> commentNodes = httpPage.getHtml().xpath("//div[@class=\"cmtcontent\"]/div/div[@id]").nodes();
        Collections.reverse(commentNodes);
        for (Selectable commentNode : commentNodes) {
            String commentId = commentNode.xpath("./@pid").get();
            String authorId = commentNode.xpath("./@uid").get();
            String author = commentNode.xpath("./div[@class=\"cmtList-detail\"]/@data-nick").get();
            String floor = commentNode.xpath("./div[@class=\"cmtList-detail\"]/div/span[@class=\"cmtList-detail-floor\"]/text()").get().trim();
            String content = commentNode.xpath("./div[@class=\"cmtList-detail\"]/div[@class=\"cmtList-detail-commment\"]/text()").get();
            String pubTime = commentNode.xpath("./div[@class=\"cmtList-detail\"]/@data-time").get();
            String authorFrom = "";
            String postClient = "";

            Map<String, Object> cExtras = new HashMap<>();
            cExtras.put("articleKey", articleKey);
            cExtras.put("commentId", commentId);
            cExtras.put("author", author);
            cExtras.put("authorId", authorId);
            cExtras.put("authorFrom", authorFrom);
            cExtras.put("floor", floor);
            cExtras.put("content", content);
            cExtras.put("postClient", postClient);

            String commentUrl = "https://www.pcauto.com.cn/#" + commentId;
            try {
                CrawlerRequestRecord commentRecord = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRequestRecord)
                        .httpUrl(commentUrl)
                        .recordKey(commentUrl)
                        .releaseTime(DateUtils.parseDate(pubTime, "yyyy-MM-dd HH:mm:ss").getTime())
                        .copyBizTags()
                        .resultLabelTag(comment)
                        .needParsed(true)
                        .needWashed(true)
                        .build();
                commentRecord.setDownload(false);
                HttpPage page = new HttpPage();
                page.setRequest(crawlerRequestRecord.getHttpRequest());
                page.setRawText(JSON.toJSONString(cExtras));
                commentRecord.setInternalDownloadPage(page);
                commentRecord.getHttpRequest().setExtras(cExtras);
                parsedLinks.add(commentRecord);
            } catch (ParseException e) {
                logger.error("parse date error", e);
            }

        }
    }

    private void parseNormalComments(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks, String articleKey) {
        List<Selectable> commentNodes = httpPage.getHtml().xpath("//div[@id=\"post_list\"]/div[@id]").nodes();
        Collections.reverse(commentNodes);
        for (Selectable commentNode : commentNodes) {
            String nodeClass = commentNode.xpath("./@class").get();
            if (StringUtils.isBlank(nodeClass) || !nodeClass.trim().matches("postid_\\d*\\spost_wrap\\spostids")) {
                continue;
            }
            String commentId = commentNode.xpath("./table/@id").get().substring(3);
            String author = commentNode.xpath("./table/tbody/tr[1]/td[1]/div/p/a/text()").get();
            String authorUrl = commentNode.xpath("./table/tbody/tr[1]/td[1]/div/p/a/@href").get();
            String authorId = authorUrl.split("cn/")[1].split("/f")[0];
            String authorFrom = commentNode.xpath("./table/tbody/tr[1]/td[1]/div/div[@class=\"user_info\"]/ul/li/a[@class=\"dblue\"]/text()").get();
            String pubTime = commentNode.xpath("./table/tbody/tr[1]/td[2]/div/div[@class=\"post_time\"]/text()").get();
            pubTime = pubTime.replaceAll("&#\\w*;", "").replaceAll("\n", "").trim();
            if (pubTime.contains("于")) {
                pubTime = pubTime.split("于")[1].trim();
            }
            String floor = "";
            List<String> floors = commentNode.xpath("./table/tbody/tr[1]/td[2]/div/div[@class=\"post_floor\"]/em/text() | ./table/tbody/tr[1]/td[2]/div/div[@class=\"post_floor\"]/text()").all();
            for (String s : floors) {
                String f = s.replaceAll("&#\\w*;", "").replaceAll("\n", "").trim();
                if (StringUtils.isNotBlank(f)) {
                    floor = f;
                }
            }
            List<String> contents = commentNode.xpath("./table/tbody/tr/td[2]/div[2]/div/div[2]/table/tbody/tr/td[1]/div//text()").all();
            String content = "";
            for (String s : contents) {
                if (StringUtils.isNotBlank(s.trim())){
                    content = content + s.trim();
                }
            }

            content = content.replaceAll("&#\\w*;", "").replaceAll("\n", "").trim();
            String postClient = commentNode.xpath("./table/tbody/tr/td[2]/div[2]/div/div[1]/a/font/text()").get();
            if (StringUtils.isBlank(postClient)) {
                postClient = "";
            }
            Map<String, Object> cExtras = new HashMap<>();
            cExtras.put("articleKey", articleKey);
            cExtras.put("commentId", commentId);
            cExtras.put("author", author);
            cExtras.put("authorId", authorId);
            cExtras.put("authorFrom", authorFrom);
            cExtras.put("floor", floor);
            cExtras.put("content", content);
            cExtras.put("postClient", postClient);

            String commentUrl = "https://www.pcauto.com.cn/#" + commentId;
            try {
                CrawlerRequestRecord commentRecord = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRequestRecord)
                        .httpUrl(commentUrl)
                        .recordKey(commentUrl)
                        .releaseTime(DateUtils.parseDate(pubTime, "yyyy-MM-dd HH:mm").getTime())
                        .copyBizTags()
                        .resultLabelTag(comment)
                        .needParsed(true)
                        .needWashed(true)
                        .build();
                commentRecord.setDownload(false);
                HttpPage page = new HttpPage();
                page.setRequest(crawlerRequestRecord.getHttpRequest());
                page.setRawText(JSON.toJSONString(cExtras));
                commentRecord.setInternalDownloadPage(page);
                commentRecord.getHttpRequest().setExtras(cExtras);
                parsedLinks.add(commentRecord);
            } catch (ParseException e) {
                logger.error("parse date error", e);
            }
        }
    }

    private CrawlerRequestRecord getAuthorInfoRecord(CrawlerRequestRecord crawlerRequestRecord, String lastRequestUrl, String authorUrl) {
        CrawlerRequestRecord authorInfoRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRequestRecord)
                .httpUrl(authorUrl)
                .recordKey(authorUrl)
                .releaseTime(System.currentTimeMillis())
                .notFilterRecord()
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.internalDownload)
                .build();
        HttpRequest httpRequest = authorInfoRecord.getHttpRequest();
        httpRequest.addHeader("Referer", lastRequestUrl);
        httpRequest.addHeader("User-Agent", getRandomUA());
        return authorInfoRecord;
    }

    private CrawlerRequestRecord getAuthorCarInfoRecord(CrawlerRequestRecord crawlerRequestRecord, String lastRequestUrl, String authorCarUrl) {
        CrawlerRequestRecord authorInfoRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRequestRecord)
                .httpUrl(authorCarUrl)
                .recordKey(authorCarUrl)
                .releaseTime(System.currentTimeMillis())
                .notFilterRecord()
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.internalDownload)
                .build();
        HttpRequest httpRequest = authorInfoRecord.getHttpRequest();
        httpRequest.addHeader("Referer", lastRequestUrl);
        httpRequest.addHeader("User-Agent", getRandomUA());
        return authorInfoRecord;
    }

    private CrawlerRequestRecord getFollowsRecord(CrawlerRequestRecord crawlerRequestRecord, String authorUrl, String followsUrl) {
        CrawlerRequestRecord followsRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRequestRecord)
                .httpUrl(followsUrl)
                .recordKey(followsUrl)
                .releaseTime(System.currentTimeMillis())
                .notFilterRecord()
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.internalDownload)
                .build();

        HttpRequest httpRequest = followsRecord.getHttpRequest();
        httpRequest.addHeader("Referer", authorUrl);
        httpRequest.addHeader("sec-fetch-dest", "empty");
        httpRequest.addHeader("sec-fetch-mode", "cors");
        httpRequest.addHeader("sec-fetch-site", "same-origin");
        httpRequest.addHeader("User-Agent", getRandomUA());
        return followsRecord;
    }

    private CrawlerRequestRecord getViewsRecord(CrawlerRequestRecord crawlerRequestRecord, String lastRequestUrl, String viewsUrl) {
        CrawlerRequestRecord viewsRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRequestRecord)
                .httpUrl(viewsUrl)
                .recordKey(viewsUrl)
                .releaseTime(System.currentTimeMillis())
                .notFilterRecord()
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.internalDownload)
                .build();

        HttpRequest httpRequest = viewsRecord.getHttpRequest();
        httpRequest.addHeader("Host", "bbs.pcauto.com.cn");
        httpRequest.addHeader("Referer", lastRequestUrl);
        httpRequest.addHeader("X-Requested-With", "XMLHttpRequest");
        httpRequest.addHeader("Cookie", "u=50x2fzf; PClocation=3; pcsuv=1610334270311.a.653977251; c=50x2lkg; __PCautoPrice4s_area_id_=3-%u4E0A%u6D77-3; pcautoLocate=%7B%22proId%22%3A7%2C%22cityId%22%3A3%2C%22url%22%3A%22%2F%2Fwww.pcauto.com.cn%2Fqcbj%2Fsh%2F%22%2C%22dataTypeAuto%22%3A%22user%22%7D; locationWap=%7B%22isUser%22%3A0%2C%22pro%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%E7%9C%81%22%2C%22city%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22cityCode%22%3A%22310000%22%2C%22proCode%22%3A%22310000%22%2C%22cityUser%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22cityCodeUser%22%3A%22310000%22%2C%22proCodeUser%22%3A%22310000%22%2C%22regionPro%22%3A7%2C%22regionCity%22%3A3%2C%22url%22%3A%22%2F%2Fm.pcauto.com.cn%2Fx%2Fqcbj%2Fsh%2F%22%2C%22expires%22%3A1611639787528%7D; m_CMT4_IP_AREA=%E4%B8%8A%E6%B5%B7%E5%B8%82; iwt_uuid=8ba3deec-d0c5-414f-9900-bb5e9f641a14; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22176ef7676ff590-0ccbad136e030c-31346d-2073600-176ef7677008c5%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%22176ef7676ff590-0ccbad136e030c-31346d-2073600-176ef7677008c5%22%7D; pcLocate=%7B%22proCode%22%3A%22310000%22%2C%22pro%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%E7%9C%81%22%2C%22cityCode%22%3A%22310000%22%2C%22city%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22dataType%22%3A%22user%22%2C%22expires%22%3A1611817047544%7D; favCar=%E5%93%88%E5%BC%97%E5%88%9D%E6%81%8B_26834%7C%E6%9E%81%E7%8B%90%20%E9%98%BF%E5%B0%94%E6%B3%95T(ARCFOX%20%CE%B1T)_25982%7C%E5%B8%95%E8%90%A8%E7%89%B9_99%7C%E9%A2%86%E8%88%AA%E5%91%98_75%7C%E6%8D%B7%E8%B1%B9F-PACE_9968; pc_browsing_history=%7B%22id%22%3A25549%2C%20%22name%22%3A%22%E6%BD%8D%E6%9F%B4%E6%B1%BD%E8%BD%A6U70%22%7D%7C%7B%22id%22%3A26834%2C%20%22name%22%3A%22%E5%93%88%E5%BC%97%E5%88%9D%E6%81%8B%22%7D%7C%7B%22id%22%3A25982%2C%20%22name%22%3A%22%E6%9E%81%E7%8B%90%20%E9%98%BF%E5%B0%94%E6%B3%95T(ARCFOX%20%CE%B1T)%22%7D%7C%7B%22id%22%3A99%2C%20%22name%22%3A%22%E5%B8%95%E8%90%A8%E7%89%B9%22%7D%7C%7B%22id%22%3A75%2C%20%22name%22%3A%22%E9%A2%86%E8%88%AA%E5%91%98%22%7D%7C%7B%22id%22%3A9968%2C%20%22name%22%3A%22%E6%8D%B7%E8%B1%B9F-PACE%22%7D; visitedfid=14150D20200D20089D17951D16100D15303D31238D24795; captcha=f0cd3e1829426a5e-d4c1fe7176f81037372a58251084339251004326; pcuvdata=lastAccessTime=1610531654296|visits=39; channel=6609");
        httpRequest.addHeader("User-Agent", getRandomUA());
        return viewsRecord;
    }

    private List<CrawlerRequestRecord> parseListLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks) {
        HttpRequest lastRequest = crawlerRequestRecord.getHttpRequest();
        String lastRequestUrl = lastRequest.getUrl();
        Map<String, Object> listExtras = lastRequest.getExtras();
        String nowPage = httpPage.getHtml().xpath("//link[@rel=\"alternate\"]/@href").get();
        String forumKey = nowPage.substring(nowPage.lastIndexOf("forum-") + 6).split("\\.")[0];
        if (forumKey.contains("-")){
            forumKey = forumKey.split("-")[0];
        }
        listExtras.put("forumKey",forumKey);
        int forumPage = (int) listExtras.get("forumPage");
        //判断翻页
        String url = httpPage.getHtml().xpath("//a[@class=\"next\"]/@href").get();
        if (StringUtils.isNotBlank(url)) {
            String nextPageUrl = String.format(listUrlFormat, forumKey, (forumPage + 1));
            CrawlerRequestRecord nextPageRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRequestRecord)
                    .httpUrl(nextPageUrl)
                    .recordKey(nextPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .build();
            Map<String, Object> nextListExtras = copyExtras(listExtras);
            nextListExtras.put("forumPage", (forumPage + 1));
            nextListExtras.put("downloadTimes", 1);
            nextPageRecord.getHttpRequest().setExtras(nextListExtras);

            HttpRequest httpRequest = nextPageRecord.getHttpRequest();
            httpRequest.addHeader("Host", "bbs.pcauto.com.cn");
            httpRequest.addHeader("User-Agent", getRandomUA());
            parsedLinks.add(nextPageRecord);
        }
        String forumName = httpPage.getHtml().xpath("//div[@id=\"forumInfo\"]//span[@class=\"mark\"]/text()").get();
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(Tag_Field_Forum_Name, forumName);
        //解析列表
//        boolean flag = true;
        List<Selectable> itemNodes = httpPage.getHtml().xpath("//div[@id=\"topic_list\"]/form/table/tbody/tr[@onmouseover=\"style.backgroundColor='#F4F4F4'\"]").nodes();
        for (Selectable itemNode : itemNodes) {
//            if (flag) {
//                String blockName = itemNode.xpath("./tr/td/text()").get();
//                if (StringUtils.isNotBlank(blockName) && blockName.contains("版块主题")) {
//                    flag = false;
//                }
//                continue;
//            }
            String itemUrl = itemNode.xpath("./th/span[@class=\"checkbox_title\"]/a/@href").get();
            if (StringUtils.isBlank(itemUrl)) {
                continue;
            }
            String articleKey = itemUrl.split("-")[1].split("\\.")[0];
            itemUrl = String.format(articleUrlFormat, articleKey, 1);
            String pubTime = itemNode.xpath("./td[@class=\"lastpost\"]/em/text()").get();
            if (pubTime.startsWith("-")) {
                pubTime = pubTime.split("-")[1];
            }
            try {
                long releaseTime = cleanTime(pubTime);
                CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRequestRecord)
                        .httpUrl(itemUrl)
                        .recordKey(itemUrl)
                        .releaseTime(releaseTime)
                        .copyBizTags()
                        .resultLabelTag(article)
                        .resultLabelTag(interaction)
                        .build();
                Map<String, Object> itemExtras = copyExtras(listExtras);
                itemExtras.put("listUrl", lastRequestUrl);
                itemExtras.put("articleUrl", itemUrl);
                itemExtras.put("articleKey", articleKey);
                itemExtras.put("articlePage", 1);
                itemExtras.put("downloadTimes", 1);
                itemRecord.getHttpRequest().setExtras(itemExtras);

                HttpRequest itemRequest = itemRecord.getHttpRequest();
                itemRequest.addHeader("Host", "bbs.pcauto.com.cn");
                itemRequest.addHeader("Referer", lastRequestUrl);
                itemRequest.addHeader("User-Agent", getRandomUA());
                itemRequest.addHeader("Cookie", "u=50x2fzf; pcsuv=1610334270311.a.653977251; c=50x2lkg; locationWap=%7B%22isUser%22%3A0%2C%22pro%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%E7%9C%81%22%2C%22city%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22cityCode%22%3A%22310000%22%2C%22proCode%22%3A%22310000%22%2C%22cityUser%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22cityCodeUser%22%3A%22310000%22%2C%22proCodeUser%22%3A%22310000%22%2C%22regionPro%22%3A7%2C%22regionCity%22%3A3%2C%22url%22%3A%22%2F%2Fm.pcauto.com.cn%2Fx%2Fqcbj%2Fsh%2F%22%2C%22expires%22%3A1611639787528%7D; m_CMT4_IP_AREA=%E4%B8%8A%E6%B5%B7%E5%B8%82; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22176ef7676ff590-0ccbad136e030c-31346d-2073600-176ef7677008c5%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%22176ef7676ff590-0ccbad136e030c-31346d-2073600-176ef7677008c5%22%7D; UM_distinctid=1771f5052622e9-0033ca8f8137a9-31346d-1fa400-1771f5052638bc; PClocation=3; pcautoLocate=%7B%22proId%22%3A7%2C%22cityId%22%3A3%2C%22url%22%3A%22%2F%2Fwww.pcauto.com.cn%2Fqcbj%2Fsh%2F%22%2C%22dataTypeAuto%22%3A%22user%22%7D; pcLocate=%7B%22proCode%22%3A%22310000%22%2C%22pro%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%E7%9C%81%22%2C%22cityCode%22%3A%22310000%22%2C%22city%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22dataType%22%3A%22user%22%2C%22expires%22%3A1615260185126%7D; iwt_uuid=8ba3deec-d0c5-414f-9900-bb5e9f641a14; __PCautoPrice4s_area_id_=3-%u4E0A%u6D77-3; captcha=102515f6402dde214758fd1b177f5a971c4-33c38080228949961102; favCar=%E4%B8%8A%E6%B1%BD%E5%A4%A7%E9%80%9AMAXUS%20V80_7590%7C%E4%B8%8A%E6%B1%BD%E5%A4%A7%E9%80%9AMAXUS%20V80%E6%96%B0%E8%83%BD%E6%BA%90_22696%7C%E9%A3%8E%E9%AA%8F%E6%88%BF%E8%BD%A6_12874%7C%E6%8E%A2%E5%B2%B3_22813%7C%E6%9C%97%E9%80%B8_3225; 101362=%7B%22id%22%3A3225%2C%20%22name%22%3A%22%E6%9C%97%E9%80%B8%22%7D; visitedfid=16040D23855D27445D21855D25455D15820D32236D32335D14150D17557; ztLocate=%7B%22proId%22%3A7%2C%22cityId%22%3A3%2C%22proCode%22%3A%22310000%22%2C%22cityCode%22%3A%22310000%22%2C%22url%22%3A%22%2F%2Fwww.pcauto.com.cn%2Fqcbj%2Fsh%2F%22%2C%22proLetter%22%3A%22S%22%2C%22area%22%3A%22%E5%8D%8E%E4%B8%9C%22%2C%22near%22%3A%22%E5%8D%8E%E4%B8%9C%22%2C%22cityLetter%22%3A%22S%22%2C%22name%22%3A%22%E4%B8%8A%E6%B5%B7%22%2C%22pro%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22core%22%3Atrue%2C%22proPinyin%22%3A%22shanghaishi%22%2C%22pinyin%22%3A%22shanghai%22%2C%22city%22%3A%22%E4%B8%8A%E6%B5%B7%22%7D; channel=1067; pcuvdata=lastAccessTime=1614749730190|visits=148");
                parsedLinks.add(itemRecord);
            } catch (Exception e) {
                logger.error("parse date error", e);
            }
        }
        return parsedLinks;
    }

    private List<CrawlerRequestRecord> parseHomeLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerRequestRecord> parsedLinks, String lastRequestUrl) {
        Html html = httpPage.getHtml();
        String forumUrl = html.xpath("//span[text()='论坛']/../@href").get();
        if (StringUtils.isNotBlank(forumUrl)) {
            if (forumUrl.contains("forum-20995.")) {
                forumUrl = forumUrl.replace("20995", "20851");
            }
        }else {
            logger.error("parse forum url failed : {}",lastRequestUrl);
        }
        String forumKey = forumUrl.split("-")[1].split("\\.")[0];
//        if (forumKey.equals("18746")) {
//            forumKey = "17323";
//        }
//        if (forumKey.equals("20877")) {
//            forumKey = "20851";
//        }
//        if (forumKey.equals("27886")) {
//            forumKey = "21205";
//        }
        forumUrl = String.format(listUrlFormat, forumKey, 1);
        Map<String, Object> extras = new HashMap<>();
        extras.put("forumKey", forumKey);
        extras.put("forumPage", 1);
        extras.put("downloadTimes", 1);
        String brand = html.xpath("//div[@class=\"pos-mark\"]/a[4]/text()").get();
        String seriesName = html.xpath("//div[@class=\"pos-mark\"]/a[5]/text()").get();
        String seriesId = lastRequestUrl.split("cn/")[1].replace("/", "");
        CrawlerRequestRecord forumIndexRecord = CrawlerRequestRecord.builder()
                .turnPageRequest(crawlerRequestRecord)
                .httpUrl(forumUrl)
                .recordKey(forumUrl)
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .notFilterRecord()
                .build();
        List<Map<String, String>> series = new ArrayList<>();
        Map<String, String> seriesInfo = new HashMap<>();
        seriesInfo.put("series_name", seriesName);
        seriesInfo.put("series_url", lastRequestUrl);
        seriesInfo.put("series_id", seriesId);
        series.add(seriesInfo);
        forumIndexRecord.tagsCreator().bizTags().addCustomKV(Tag_Field_Series, series);
        forumIndexRecord.tagsCreator().bizTags().addCustomKV(Tag_Field_Brand, brand);
        forumIndexRecord.getHttpRequest().setExtras(extras);

        HttpRequest httpRequest = forumIndexRecord.getHttpRequest();
        httpRequest.addHeader("Host", "bbs.pcauto.com.cn");
        httpRequest.addHeader("User-Agent", getRandomUA());
        httpRequest.addHeader("Referer", "https://bbs.pcauto.com.cn/");
        httpRequest.addHeader("Cookie", "u=50x2fzf; pcsuv=1610334270311.a.653977251; c=50x2lkg; locationWap=%7B%22isUser%22%3A0%2C%22pro%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%E7%9C%81%22%2C%22city%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22cityCode%22%3A%22310000%22%2C%22proCode%22%3A%22310000%22%2C%22cityUser%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22cityCodeUser%22%3A%22310000%22%2C%22proCodeUser%22%3A%22310000%22%2C%22regionPro%22%3A7%2C%22regionCity%22%3A3%2C%22url%22%3A%22%2F%2Fm.pcauto.com.cn%2Fx%2Fqcbj%2Fsh%2F%22%2C%22expires%22%3A1611639787528%7D; m_CMT4_IP_AREA=%E4%B8%8A%E6%B5%B7%E5%B8%82; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22176ef7676ff590-0ccbad136e030c-31346d-2073600-176ef7677008c5%22%2C%22first_id%22%3A%22%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22%24device_id%22%3A%22176ef7676ff590-0ccbad136e030c-31346d-2073600-176ef7677008c5%22%7D; UM_distinctid=1771f5052622e9-0033ca8f8137a9-31346d-1fa400-1771f5052638bc; pcautoLocate=%7B%22proId%22%3A7%2C%22cityId%22%3A3%2C%22url%22%3A%22%2F%2Fwww.pcauto.com.cn%2Fqcbj%2Fsh%2F%22%2C%22dataTypeAuto%22%3A%22user%22%7D; PClocation=3; pcLocate=%7B%22proCode%22%3A%22310000%22%2C%22pro%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%E7%9C%81%22%2C%22cityCode%22%3A%22310000%22%2C%22city%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22dataType%22%3A%22user%22%2C%22expires%22%3A1615260185126%7D; iwt_uuid=8ba3deec-d0c5-414f-9900-bb5e9f641a14; __PCautoPrice4s_area_id_=3-%u4E0A%u6D77-3; 101362=%7B%22id%22%3A3225%2C%20%22name%22%3A%22%E6%9C%97%E9%80%B8%22%7D; visitedfid=17613D32147D32335D31249D19557D32247D31736D30795D18555D23855; captcha=d7fe78ae42e4d97e-1fbe8135177f28a535b3ead162116361249139990; favCar=%E9%A3%8E%E9%AA%8F%E6%88%BF%E8%BD%A6_12874%7C%E6%8E%A2%E5%B2%B3_22813%7C%E6%9C%97%E9%80%B8_3225%7C%E5%93%AA%E5%90%92U_24455%7C%E9%A9%AC%E8%87%AA%E8%BE%BECX-5_9306; pc_browsing_history=%7B%22id%22%3A24455%2C%20%22name%22%3A%22%E5%93%AA%E5%90%92U%22%7D%7C%7B%22id%22%3A9306%2C%20%22name%22%3A%22%E9%A9%AC%E8%87%AA%E8%BE%BECX-5%22%7D; ztLocate=%7B%22proId%22%3A7%2C%22cityId%22%3A3%2C%22proCode%22%3A%22310000%22%2C%22cityCode%22%3A%22310000%22%2C%22url%22%3A%22%2F%2Fwww.pcauto.com.cn%2Fqcbj%2Fsh%2F%22%2C%22proLetter%22%3A%22S%22%2C%22area%22%3A%22%E5%8D%8E%E4%B8%9C%22%2C%22near%22%3A%22%E5%8D%8E%E4%B8%9C%22%2C%22cityLetter%22%3A%22S%22%2C%22name%22%3A%22%E4%B8%8A%E6%B5%B7%22%2C%22pro%22%3A%22%E4%B8%8A%E6%B5%B7%E5%B8%82%22%2C%22core%22%3Atrue%2C%22proPinyin%22%3A%22shanghaishi%22%2C%22pinyin%22%3A%22shanghai%22%2C%22city%22%3A%22%E4%B8%8A%E6%B5%B7%22%7D; channel=6609; pcuvdata=lastAccessTime=1614755354330|visits=153");
        parsedLinks.add(forumIndexRecord);
        return parsedLinks;
    }

    @Override
    public void afterInternalDownload(CrawlerRequestRecord crawlerRecord, List<CrawlerRequestRecord> internalDownloadRecords, List<CrawlerRequestRecord> links) {
        HttpRequest httpRequest = crawlerRecord.getHttpRequest();
        Map<String, Object> extras = httpRequest.getExtras();
        for (CrawlerRequestRecord internalDownloadRecord : internalDownloadRecords) {
            HttpRequest internalRequest = internalDownloadRecord.getHttpRequest();
            String internalRequestUrl = internalRequest.getUrl();
            HttpPage internalDownloadPage = internalDownloadRecord.getInternalDownloadPage();
            if (internalDownloadPage.isDownloadSuccess() && internalDownloadPage.getStatusCode() == 200) {
                if (internalRequestUrl.matches(viewsRegex)) {
                    try {
                        JSONObject pageObj = JSONObject.parseObject(internalDownloadPage.getRawText());
                        String views = pageObj.getString("views");
                        extras.put("views", views);
                    } catch (Exception e) {
                        extras.put("views", "0");
                    }
                }
                if (internalRequestUrl.matches(authorInfoRegex)) {
                    try {
                        String infoStr = "";
                        Matcher mtInfo = Pattern.compile("loadBBSForum\\(\\{.*\\}\\);").matcher(internalDownloadPage.getRawText());
                        while (mtInfo.find()) {
                            infoStr = mtInfo.group(0).split("\\(")[1].split("\\)")[0];
                        }
                        JSONObject pageObj = JSONObject.parseObject(infoStr);
                        String signTime = pageObj.getString("createAt");
                        String topicCount = pageObj.getString("topicCount");
                        String postCount = pageObj.getString("postCount");
                        String pickCount = pageObj.getString("pickCount");
                        String replyCount = String.valueOf(Integer.parseInt(postCount) - Integer.parseInt(topicCount));
                        extras.put("signTime", signTime);
                        extras.put("topicCount", topicCount);
                        extras.put("replyCount", replyCount);
                        extras.put("pickCount", pickCount);
                    } catch (Exception e) {
                        extras.put("signTime", "");
                        extras.put("topicCount", "0");
                        extras.put("replyCount", "0");
                        extras.put("pickCount", "0");
                    }
                }
                if (internalRequestUrl.matches(followsRegex)) {
                    String follows = internalDownloadPage.getRawText().trim();
                    extras.put("follows", follows);
                }
                if (internalRequestUrl.matches(authorCarInfoRegex)){
                    try {
                        JSONObject pageObj = JSONObject.parseObject(internalDownloadPage.getRawText());
                        JSONObject vipInfo = pageObj.getJSONArray("vipInfo").getJSONObject(0);
                        String serialName = vipInfo.getString("serialname");
                        String modelName = vipInfo.getString("modelname");
                        String identification = serialName + " " + modelName;
                        extras.put("identification",identification);
                    } catch (Exception e) {
                        extras.put("identification","");
                    }
                }
            } else {
                crawlerRecord.setNeedWashPage(false);
                crawlerRecord.setNeedParsedPage(false);
                CrawlerRequestRecord newRecord = CrawlerRequestRecord.builder()
                        .itemPageRequest(crawlerRecord)
                        .httpUrl(crawlerRecord.getHttpRequest().getUrl())
                        .recordKey(crawlerRecord.getHttpRequest().getUrl())
                        .releaseTime(crawlerRecord.getReleaseTime())
                        .copyBizTags()
                        .needParsed(true)
                        .needWashed(true)
                        .notFilterRecord()
                        .build();
                newRecord.getHttpRequest().setExtras(extras);
                links.add(newRecord);
            }
        }
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        if (crawlerRequestRecord.tagsCreator().resultTags().hasDataType(article)) {
            crawlerDataList.addAll(washArticle(crawlerRequestRecord, httpPage));

        }
        if (crawlerRequestRecord.tagsCreator().resultTags().hasDataType(interaction)) {
            crawlerDataList.add(washInteraction(crawlerRequestRecord, httpPage));
        }
        if (crawlerRequestRecord.tagsCreator().resultTags().hasDataType(comment)) {
            crawlerDataList.addAll(washComment(crawlerRequestRecord, httpPage));
        }
        return crawlerDataList;
    }

    public List<CrawlerData> washArticle(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        HttpRequest httpRequest = crawlerRequestRecord.getHttpRequest();
        Map<String, Object> extras = httpRequest.getExtras();
        Html html = httpPage.getHtml();
        String articleKey = (String) extras.get("articleKey");
        String banner = html.xpath("//div[@id=\"Jbanner\"]/img/@src").get();
        if (StringUtils.isBlank(banner)) {
            washNormalArticle(crawlerRequestRecord, httpPage, crawlerArticleDataList);
        } else {
            washNewArticle(crawlerRequestRecord, httpPage, crawlerArticleDataList);
        }


        return crawlerArticleDataList;
    }

    private void washNormalArticle(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerData> crawlerArticleDataList) {
        HttpRequest httpRequest = crawlerRequestRecord.getHttpRequest();
        String lastRequestUrl = httpRequest.getUrl();
        Map<String, Object> extras = httpRequest.getExtras();
        Html html = httpPage.getHtml();
        String articleKey = (String) extras.get("articleKey");

        String follows = (String) extras.get("follows");
        String signTime = (String) extras.get("signTime");
        String topicCount = (String) extras.get("topicCount");
        String replyCount = (String) extras.get("replyCount");
        String pickCount = (String) extras.get("pickCount");
        String identification = (String) extras.get("identification");

        String author = html.xpath("//div[@id=\"post_list\"]/div[@id][1]/table/tbody/tr[1]/td[1]/div/p/a/text()").get();
        String authorUrl = html.xpath("//div[@id=\"post_list\"]/div[@id][1]/table/tbody/tr[1]/td[1]/div/p/a/@href").get();
        String authorId = null;
        try {
            authorId = authorUrl.split("cn/")[1].split("/f")[0];
        } catch (Exception e) {
            authorId = "";
        }
        String authorFrom = html.xpath("//div[@id=\"post_list\"]/div[@id][1]/table/tbody/tr[1]/td[1]/div/div[@class=\"user_info\"]/ul/li/a[@class=\"dblue\"]/text()").get();

        String floor = html.xpath("//div[@id=\"post_list\"]/div[@id][1]/table/tbody/tr[1]/td[2]/div//em[@class=\"floor1\"]/text()").get();
        String pubTime = html.xpath("//div[@id=\"post_list\"]/div[@id][1]/table/tbody/tr[1]/td[2]/div/div[@class=\"post_time\"]/text()").get();
        pubTime = pubTime.replaceAll("&#\\w*;", "").replaceAll("\n", "").trim();
        if (pubTime.contains("于")) {
            pubTime = pubTime.split("于")[1].trim();
        }
        List<String> contents = html.xpath("//div[@id=\"post_list\"]/div[@id][1]/table/tbody/tr[1]/td[2]/div[@id]/div/div/table/tbody/tr/td[1]/div[1]//text()").all();
        StringBuffer sbContent = new StringBuffer();
        for (String text : contents) {
            sbContent.append(text);
        }
        List<String> images = html.xpath("//div[@id=\"post_list\"]/div[@id][1]/table/tbody/tr[1]/td[2]/div[@id]/div/div/table/tbody/tr/td[1]/div[1]//img/@src").all();
        StringBuffer sbImage = new StringBuffer();
        for (String image : images) {
            sbImage.append(image).append("\\x01");
        }
        String title = html.xpath("//div[@class=\"post_list_top\"]//h1//text()").get();
        title = unescapeHtml2J(title);
        Selectable el = html.xpath("//div[@id=\"post_list\"]/div[@id][1]/table/tbody/tr[1]/td[2]/div[@class=\"pick1\"]");
        String isElite = "";
        if (el.nodes().size() > 0) {
            isElite = "是";
        } else {
            isElite = "否";
        }

        try {
            String site = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
            CrawlerData crawlerArticleData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), site, article.enumVal(), articleKey))
                    .url(lastRequestUrl)
                    .releaseTime(DateUtils.parseDate(pubTime, "yyyy-MM-dd HH:mm").getTime())
                    .addContentKV(Field_Author, author)
                    .addContentKV(Field_Author_Id, authorId)
                    .addContentKV(Field_Title, title)
                    .addContentKV(Field_Content, sbContent.toString().replaceAll("&#\\w*;", "").replaceAll("\n", "").trim())
                    .addContentKV(Field_Author_Topic_Count, topicCount)
                    .addContentKV(Field_Author_Pick_Count, pickCount)
                    .addContentKV(Field_Author_Reply_Count, replyCount)
                    .addContentKV(Field_Author_Follows, follows)
                    .addContentKV(Field_Author_Identification_Model, identification)
                    .addContentKV(Field_Author_From, authorFrom)
                    .addContentKV(Field_Author_Sign_In, signTime)
                    .addContentKV(Field_Floor, floor)
                    .addContentKV(Field_Images, sbImage.toString().trim())
                    .resultLabelTag(article)
                    .build();
            crawlerArticleData.tagsCreator().bizTags().addSiteBiz("forum");
            crawlerArticleData.tagsCreator().bizTags().addCustomKV(Tag_Field_Is_Elite, isElite);
            crawlerArticleDataList.add(crawlerArticleData);
        } catch (Exception e) {
            logger.error("parse date error", e);
        }
    }

    private void washNewArticle(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage, List<CrawlerData> crawlerArticleDataList) {
        HttpRequest httpRequest = crawlerRequestRecord.getHttpRequest();
        String lastRequestUrl = httpRequest.getUrl();
        Map<String, Object> extras = httpRequest.getExtras();
        Html html = httpPage.getHtml();
        String articleKey = (String) extras.get("articleKey");

        String follows = (String) extras.get("follows");
        String signTime = (String) extras.get("signTime");
        String topicCount = (String) extras.get("topicCount");
        String replyCount = (String) extras.get("replyCount");
        String pickCount = (String) extras.get("pickCount");

        String author = html.xpath("//div[@class=\"user-name\"]/a/text()").get();
        String authorUrl = html.xpath("//div[@class=\"user-name\"]/a/@href").get();
        String authorId = authorUrl.split("cn/")[1].split("/t")[0];
        String authorFrom = html.xpath("//span[@class=\"cityshow\"]/text()").get();
        String identification = (String) extras.get("identification");

        String floor = "楼主";
        String pubTime = html.xpath("//span[@class=\"post-time\"]/text()").get();
        List<String> contents = html.xpath("//div[@class=\"post-text\"]/div//text()").all();
        StringBuffer sbContent = new StringBuffer();
        for (String text : contents) {
            sbContent.append(text);
        }
        List<String> images = html.xpath("//div[@class=\"post-text\"]/div//img/@src").all();
        StringBuffer sbImage = new StringBuffer();
        for (String image : images) {
            sbImage.append(image).append("\\x01");
        }
        String title = html.xpath("//div[@class=\"t-title\"]/text()").get();
        Selectable el = html.xpath("class=\"pick2\"");
        String isElite = "";
        if (el.nodes().size() > 0) {
            isElite = "是";
        } else {
            isElite = "否";
        }

        try {
            String site = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
            CrawlerData crawlerArticleData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), site, article.enumVal(), articleKey))
                    .url(lastRequestUrl)
                    .releaseTime(DateUtils.parseDate(pubTime, "yyyy-MM-dd HH:mm").getTime())
                    .addContentKV(Field_Author, author)
                    .addContentKV(Field_Author_Id, authorId)
                    .addContentKV(Field_Title, title)
                    .addContentKV(Field_Content, sbContent.toString().replaceAll("&#\\w*;", "").replaceAll("\n", "").trim())
                    .addContentKV(Field_Author_Topic_Count, topicCount)
                    .addContentKV(Field_Author_Pick_Count, pickCount)
                    .addContentKV(Field_Author_Reply_Count, replyCount)
                    .addContentKV(Field_Author_Follows, follows)
                    .addContentKV(Field_Author_Identification_Model, identification)
                    .addContentKV(Field_Author_From, authorFrom)
                    .addContentKV(Field_Author_Sign_In, signTime)
                    .addContentKV(Field_Floor, floor)
                    .addContentKV(Field_Images, sbImage.toString().trim())
                    .resultLabelTag(article)
                    .build();
            crawlerArticleData.tagsCreator().bizTags().addSiteBiz("forum");
            crawlerArticleData.tagsCreator().bizTags().addCustomKV(Tag_Field_Is_Elite, isElite);
            crawlerArticleDataList.add(crawlerArticleData);
        } catch (Exception e) {
            logger.error("parse date error", e);
        }
    }

    public CrawlerData washInteraction(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {

        HttpRequest lastRequest = crawlerRequestRecord.getHttpRequest();
        try {
            Map<String, Object> extras = lastRequest.getExtras();
            String topicId = (String) extras.get("articleKey");
            Html html = httpPage.getHtml();
            String comments = html.xpath("//p[@class=\"overView\"]/span[2]/text() | //li[@class=\"p-reply\"]/text()").get();
            String views = (String) extras.get("views");
            String pubTime = "";
            try {
                pubTime = html.xpath("//div[@id=\"post_list\"]/div[@id][1]/table/tbody/tr[1]/td[2]/div/div[@class=\"post_time\"]/text()").get();
                pubTime = pubTime.replaceAll("&#\\w*;", "").replaceAll("\n", "").trim();
                if (pubTime.contains("于")) {
                    pubTime = pubTime.split("于")[1].trim();
                }
            } catch (Exception e) {
                pubTime = html.xpath("//span[@class=\"post-time\"]/text()").get();
            }
            String site = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), site, interaction.enumVal(), topicId))
                    .parentId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), site, article.enumVal(), topicId))
                    .url(lastRequest.getUrl())
                    .releaseTime(DateUtils.parseDate(pubTime, "yyyy-MM-dd HH:mm").getTime())
                    .addContentKV(Field_I_Comments, comments)
                    .addContentKV(Field_I_Views, views)
                    .resultLabelTag(interaction)
                    .build();
            crawlerData.tagsCreator().bizTags().addSiteBiz("forum");
            return crawlerData;
        } catch (Exception e) {
            logger.error(e.getMessage(), e);
            return null;
        }
    }

    public List<CrawlerData> washComment(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerData> crawlerCommentDataList = new ArrayList<>();
        HttpRequest httpRequest = crawlerRequestRecord.getHttpRequest();
        Map<String, Object> extras = httpRequest.getExtras();
//        logger.info("test print extras size : {}", extras.size());
        String articleKey = (String) extras.get("articleKey");

        String follows = (String) extras.get("follows");
        String signTime = (String) extras.get("signTime");
        String topicCount = (String) extras.get("topicCount");
        String replyCount = (String) extras.get("replyCount");
        String pickCount = (String) extras.get("pickCount");

        String commentId = (String) extras.get("commentId");
        String author = (String) extras.get("author");
        String authorId = (String) extras.get("authorId");
        String authorFrom = (String) extras.get("authorFrom");
        String identification = (String) extras.get("identification");
        String floor = (String) extras.get("floor");
        String content = (String) extras.get("content");
        String postClient = (String) extras.get("postClient");
        String site = crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        CrawlerData crawlerCommentData = CrawlerData.builder()
                .data(crawlerRequestRecord, httpPage)
                .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), site, comment.enumVal(), commentId))
                .parentId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), site, article.enumVal(), articleKey))
                .url(String.format(articleUrlFormat, articleKey, 1))
                .releaseTime(crawlerRequestRecord.getReleaseTime())
                .addContentKV(Field_Author, author)
                .addContentKV(Field_Author_Id, authorId)
                .addContentKV(Field_Content, content)
                .addContentKV(Field_Author_Topic_Count, topicCount)
                .addContentKV(Field_Author_Pick_Count, pickCount)
                .addContentKV(Field_Author_Reply_Count, replyCount)
                .addContentKV(Field_Author_Follows, follows)
                .addContentKV(Field_Author_Identification_Model, identification)
                .addContentKV(Field_Author_From, authorFrom)
                .addContentKV(Field_Author_Sign_In, signTime)
                .addContentKV(Field_Floor, floor)
                .resultLabelTag(comment)
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                .build();
        crawlerCommentData.tagsCreator().bizTags().addSiteBiz("forum");
        crawlerCommentData.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
        crawlerCommentData.tagsCreator().bizTags().addCustomKV(Tag_Field_PostClient, postClient);
        crawlerCommentDataList.add(crawlerCommentData);
        return crawlerCommentDataList;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }

    public static Map<String, Object> copyExtras(Map<String, Object> inExtras) {
        Map<String, Object> extras = new HashMap<>();
        for (Map.Entry<String, Object> entry : inExtras.entrySet()) {
            extras.put(entry.getKey(), entry.getValue());
        }
        return extras;
    }

    private static List<String> agentList = new ArrayList<>();

    static {
        agentList.add("Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36");
        agentList.add("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.1");
        agentList.add("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/536.6");
        agentList.add("Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/536.6");
        agentList.add("Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.1");
        agentList.add("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)");
        agentList.add("Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)");
        agentList.add("Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)");
        agentList.add("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) ChromePlus/4.0.222.3 Chrome/4.0.222.3 Safari/532.2");
        agentList.add("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.28.3 (KHTML, like Gecko) Version/3.2.3 ChromePlus/4.0.222.3 Chrome/4.0.222.3 Safari/525.28.3");
        agentList.add("Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16");
        agentList.add("Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14");
        agentList.add("Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14");
        agentList.add("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14");
        agentList.add("Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02");
        agentList.add("Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00");
        agentList.add("Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00");
        agentList.add("Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00");
        agentList.add("Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00");
        agentList.add("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1");
        agentList.add("Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0");
        agentList.add("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0");
        agentList.add("Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0");
        agentList.add("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0");
        agentList.add("Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0");
        agentList.add("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.13 Safari/537.36");
        agentList.add("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3756.400 QQBrowser/10.5.4043.400");
    }

    private static String getRandomUA() {
        return agentList.get(RandomUtils.nextInt(0, agentList.size() - 1));
    }

    private long cleanTime(String timeStr) throws ParseException {
        if (timeStr.contains("发布")) {
            timeStr = timeStr.split("：")[1];
        }
        if (timeStr.contains("刚刚")) {
            return System.currentTimeMillis() - 1000;
        } else if (timeStr.contains("前")) {
            return timeBefore(timeStr);
        } else if (timeStr.contains("-")) {
            return DateUtils.parseDate(timeStr, "yyyy-MM-dd HH:mm").getTime();
        } else {
            return 0;
        }

    }

    private long timeBefore(String timeStr) {
        if (timeStr.matches("\\d*天前")) {
            int timeNum = Integer.parseInt(timeStr.split("天")[0]);
            return System.currentTimeMillis() - (timeNum * ONE_DAY);

        } else if (timeStr.matches("\\d*秒前")) {
            int timeNum = Integer.parseInt(timeStr.split("秒")[0]);
            return System.currentTimeMillis() - (timeNum * ONE_SECOND);

        } else if (timeStr.matches("\\d*分钟前")) {
            int timeNum = Integer.parseInt(timeStr.split("分钟")[0]);
            return System.currentTimeMillis() - (timeNum * ONE_MINUTE);

        } else if (timeStr.matches("\\d*分钟\\d*秒前")) {
            String[] split = timeStr.split("分钟");
            int minutes = Integer.parseInt(split[0]);
            int seconds = Integer.parseInt(split[1].split("秒")[0]);
            long times = (minutes * ONE_MINUTE) + (seconds * ONE_SECOND);
            return System.currentTimeMillis() - times;

        } else if (timeStr.matches("\\d*小时前")) {
            int timeNum = Integer.parseInt(timeStr.split("小时")[0]);
            return System.currentTimeMillis() - (timeNum * ONE_HOUR);

        } else if (timeStr.matches("\\d*小时\\d*分钟前")) {
            String[] split = timeStr.split("小时");
            int hours = Integer.parseInt(split[0]);
            int minutes = Integer.parseInt(split[1].split("分钟")[0]);
            long times = (hours * ONE_HOUR) + (minutes * ONE_MINUTE);
            return System.currentTimeMillis() - times;

        } else if (timeStr.matches("\\d*周前")) {
            int timeNum = Integer.parseInt(timeStr.split("周")[0]);
            return System.currentTimeMillis() - (timeNum * ONE_DAY * 7);

        } else if (timeStr.matches("\\d*个月前")) {
            int timeNum = Integer.parseInt(timeStr.split("个")[0]);
            return System.currentTimeMillis() - (timeNum * ONE_DAY * 31);
        } else {
            return 0;
        }
    }

    public static String unescapeHtml2J(String str){
        int times = 0;
        while (str.contains("&") && str.contains(";")){
            str = StringEscapeUtils.unescapeHtml(str);
            times ++;
            if (times > 5){
                break;
            }
        }
        return str;
    }
}
